from tqdm import tqdm
import jsonlines
from data import read_list_config
from processing import *


def main(file_path,output_path,simlar,black_list):
    rets = []
    with jsonlines.open(file_path) as reader:

        for obj in tqdm(reader):
            # 这里可以对每个JSON对象进行处理
            prompts = []
            ret = []
            js_li =  obj['contents']

            for i in tqdm(range(len(js_li))):
                if i<len(js_li)-1 and  i %2 == 0 and (js_li[i+1] not in black_list): prompts.append(js_li[i].strip())
            tmp2 = remove_contained_strings(list(set(prompts)))
            show(f'prompts len is {len(js_li)} | after_solve len is {len(tmp2)} | reduce  {len(prompts) - len(tmp2)} duplicate samples')
            dictionary = tmp2

            # 您提供的代码片段正在迭代“dictionary”列表中的每个项目，对于每个项目，它正在迭代“js_li”列表的索引。
            for item in dictionary:
                tmp3 = {}
                for j in (range(len(js_li))):
                    if item in js_li[j] and j+1 < len(js_li):
                        tmp3[item] =  js_li[j+1]
                        ret.append(tmp3)
            show('**********',bar=False)

            for item in ret:
                print(item)
                print(' ')
            show('**********',bar=False)
            rets.append(ret)

    show('相似度去重',bar=False)
    rets_drop_duplicate2 = remove_duplicates_by_strkey(rets)
    rets_drop_duplicate = remove_simlarity_duplicates(rets_drop_duplicate2,simlar)
    rets_drop_duplicate = remove_duplicates_from_sub(rets_drop_duplicate)
    ret,avg = converter(rets_drop_duplicate)
    ret = rm_prefix(ret)
    final = drop_duplicate_byset(ret)

    aft_avg = sum([len(x) for x in final]) / len(final)
    show(f'对话压平去重 avg : {aft_avg:.2f}  | 去除了{(((avg - aft_avg)/avg )*100):.2f}%')
    merge_lists_to_jsonl(final,output_path)

    show('summary',bar=False)
    show(f'原始数据 {len(rets)}条 | 去重 {len(rets) - len(rets_drop_duplicate)} | 剩下{len(rets_drop_duplicate)}条数据')
    

if __name__ == '__main__':
    options = {
        0: 'black_list',
        1: 'input_file', 
        2: 'output_file', 
        3: 'simlar'
        }
    black_list,inputs,output,simlar = receive_configtion('./rm.ini', 'hyperparameters',options)
    if inputs == None or inputs == '':show('输入文件的路径不合法')
    # 调用函数读取JSONL文件
    main(inputs,output,simlar,black_list)